package com.chance.cc.crawler.development.scripts.autohome.carSeriesPraise;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;

public class AutoHomeCarSeriesPraiseCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(AutoHomeCarSeriesPraiseCrawlerScript.class);//日志
    private static final String DOMAIN = "autohome";
    private static final String SITE = "carSeriesKouBei";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //   https://car.autohome.com.cn/javascript/NewSpecCompare.js?20131010
    private static final String HomeRegulars = "https://car.autohome.com.cn/javascript/NewSpecCompare.js\\?20131010";//动态链接
    //    https://club.autohome.com.cn/135
    private static final String CarSeriesRegulars = "https://www.autohome.com.cn/\\d*";//车系链接
    //    https://k.autohome.com.cn/FrontAPI/GetSpecListBySeriesId?seriesId=135&specState=1
    private static final String KouBeiRegulars = "https://k.autohome.com.cn/FrontAPI/GetSpecListBySeriesId\\?seriesId=\\d*&specState=\\d*";//口碑链接
    //    https://k.autohome.com.cn/spec/46839
    private static final String KouBeiDetailsRegulars = "https://k.autohome.com.cn/spec/\\d*";//口碑详情链接


    @Override
    public String domain() {
        return DOMAIN;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(HomeRegulars);
        addUrlRegular(CarSeriesRegulars);
        addUrlRegular(KouBeiRegulars);
        addUrlRegular(KouBeiDetailsRegulars);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type + ".msg").get();
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> keys = new ArrayList<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all) {
            JSONObject jsonObject = JSONObject.parseObject(data);
            String keyword = jsonObject.getString("keyword");
            keys.add(keyword);
        }
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("keys", keys);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();
        //        判断页面是否加载成功
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (page.getStatusCode() != 200 || (!page.isDownloadSuccess())) {
            log.error("download page url == {null} error status is {}", requestUrl, page.getStatusCode());
            if (page.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
                crawlerRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(HomeRegulars)) {
            homeRequest(crawlerRecord, page, parsedLinks);//解析动态车系
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(CarSeriesRegulars)) {
            carSeriesRequest(crawlerRecord, page, parsedLinks);//解析口碑
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(KouBeiRegulars)) {
            koubeiRequest(crawlerRecord, page, parsedLinks);//解析详情
        }
        return parsedLinks;
    }

//    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
//        Map<String, String> carMap = new HashMap<>();
//        String rawText = page.getRawText();//获取解析url后的字符串
//        String json = rawText.substring(21, rawText.length() - 3);//获取json
//        List<String> number = new Json(json).jsonPath($_type + "..I").all();
//        List<String> carName = new Json(json).jsonPath($_type + "..N").all();
//        for (int i = 0; i < number.size(); i++) {
//            carMap.put(carName.get(i), number.get(i));
//        }
//        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
//        List<String> name = (List<String>) keys.getVal();
//        ArrayList<String> list = new ArrayList<>();
//        for (int i = 0; i < name.size(); i++) {//
//            boolean discontinued = name.get(i).contains("停售");//直接进入停售页面
//            String nameNumber = carMap.get(name.get(i));//品牌转换为id数字
//            if (nameNumber == null) {
//                list.add(name.get(i));
//                log.error("没有这个车系（进口，停售等加上字段 例：开瑞K50EV(停售)）{}", name.get(i));
//            } else {
//                if (discontinued) {
//                    String discontinuedUrl = "https://www.autohome.com.cn/" + nameNumber + "/sale.html";//得到车系页面
//                    CrawlerRequestRecord discontinuedRecord = CrawlerRequestRecord.builder()//解析停售
//                            .itemPageRequest(crawlerRecord)
//                            .recordKey(discontinuedUrl)
//                            .httpUrl(discontinuedUrl)
//                            .releaseTime(System.currentTimeMillis())
//                            .resultLabelTag(article)
//                            .copyBizTags()
//                            .build();
//                    parsedLinks.add(discontinuedRecord);
//                } else {
//                    String url = "https://www.autohome.com.cn/" + nameNumber;//得到车系页面
//                    CrawlerRequestRecord record = CrawlerRequestRecord.builder()
//                            .itemPageRequest(crawlerRecord)
//                            .httpUrl(url)
//                            .releaseTime(Long.valueOf(System.currentTimeMillis()))
//                            .copyBizTags()
//                            .build();
//                    parsedLinks.add(record);
//                }
//            }
//        }
//    }

    //    解析动态车系
    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
        List<String> name = (List<String>) keys.getVal();
        for (int i = 0; i < name.size(); i++) {//name.size()
            String url = "https://www.autohome.com.cn/" + name.get(i);//得到车系页面
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .releaseTime(Long.valueOf(System.currentTimeMillis()))
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parsedLinks.add(record);
        }
    }

    private void carSeriesRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String carSeriesUrl = page.getRequest().getUrl();//车系url
        String koubeiUrl = "https:" + page.getHtml().xpath("//div[@id=\"navTop\"]//a[./text()=\"口碑\"]/@href").get();//口碑url
        String forumUrl = "https:" + page.getHtml().xpath("//div[@id=\"navTop\"]//a[./text()=\"论坛\"]/@href").get();//论坛url
        String brandName = page.getHtml().xpath("//div[@class=\"container athm-crumb\"]/a[2]/text()").get();//品牌名
        String brandUrl = "https:" + page.getHtml().xpath("//div[@class=\"container athm-crumb\"]/a[2]/@href").get();//品牌url
        String subsidiaryBrandName = page.getHtml().xpath("//div[@class=\"athm-sub-nav__car__name\"]/a/text()").get();//子品牌名
        String carSeriesName = page.getHtml().xpath("//h1/a").get();//车系名
        String carSeriesName2 = null;
        if (carSeriesName != null) {
            carSeriesName2 = subsidiaryBrandName + carSeriesName;//车系2
        }
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        HashMap<String, String> seriesMap = new HashMap<>();
        seriesMap.put(AutoVMCommonField.Field_Series_name, carSeriesName);//车系名
        seriesMap.put(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series, seriesMap);//车系
        HashMap<String, String> brandMap = new HashMap<>();
        brandMap.put(AutoVMCommonField.Field_Brand_name, brandName);//品牌名
        brandMap.put(AutoVMCommonField.Field_Brand_url, brandUrl);//品牌url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Brand, brandMap);//品牌
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Praise_url, koubeiUrl);//口碑
//        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Forum_url, forumUrl);//论坛url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series_Two, carSeriesName2);//二级车系
//        crawlerRecord.tagsCreator().bizTags().addCustomKV("subsidiaryBrandName", subsidiaryBrandName);//子品牌

        String articleId = Pattern.compile("[^0-9]").matcher(carSeriesUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
//        https://k.autohome.com.cn/FrontAPI/GetSpecListBySeriesId?seriesId=135&specState=1
        String koubeiApiUrl = "https://k.autohome.com.cn/FrontAPI/GetSpecListBySeriesId?seriesId=" + articleId + "&specState=1";

        CrawlerRequestRecord koubeiRecord = CrawlerRequestRecord.builder()//解析口碑
                .itemPageRequest(crawlerRecord)
                .recordKey(koubeiApiUrl)
                .httpUrl(koubeiApiUrl)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .build();
        parsedLinks.add(koubeiRecord);

    }

    private void koubeiRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        List<String> title = page.getJson().jsonPath($_type + "..SpecName").all();//标题
        List<String> urlId = page.getJson().jsonPath($_type + "..SpecId").all();
        if (urlId.size() > 0) {
            for (int i = 0; i < title.size(); i++) {
                crawlerRecord.tagsCreator().bizTags().addCustomKV("title", title.get(i));
                if (urlId.get(i) != null&&urlId.get(i) !="") {
                    String koubeidetailsUrl = "https://k.autohome.com.cn/spec/" + urlId.get(i);//详情url
                    CrawlerRequestRecord koubeideRecord = CrawlerRequestRecord.builder()//解析详情
                            .itemPageRequest(crawlerRecord)
                            .recordKey(koubeidetailsUrl)
                            .httpUrl(koubeidetailsUrl)
                            .releaseTime(System.currentTimeMillis())
                            .resultLabelTag(article)
                            .copyBizTags()
                            .needWashed(true)
                            .build();
                    parsedLinks.add(koubeideRecord);
                }
            }
        }
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (crawlerRecord.getHttpRequest().getUrl().matches(KouBeiDetailsRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(koubeidetailsWash(crawlerRecord, page));
            }
        }
        return crawlerDataList;
    }

    // 冲洗口碑
    public List<CrawlerData> koubeidetailsWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        Map<String, String> messageMap = new HashMap<>();
        String title = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("title");
        String koubeidetailsUrl = page.getRequest().getUrl();//口碑详情url
        String totalScore = page.getHtml().xpath("//span[@class=\"font-arial number-fen\"]/text()").get();//用户总评分
//        String number = page.getHtml().xpath("//span[@class=\"font-arial red font-16\"]/text()").get();//参与人数
        messageMap.put("用户的评分", totalScore);
//        messageMap.put("参与人数",number);
        List<String> item = page.getHtml().xpath("//div[@class=\"width-01\"]/text()").all();//项目
        List<String> score = page.getHtml().xpath("//div[@class=\"width-02\"]/text()").all();//评分
        for (int i = 0; i < item.size(); i++) {
            String it = item.get(i).trim();
            String sc = score.get(i).trim();
            String item1 = it.replaceAll("&#xD;", "").replaceAll("\\\\n", "").trim();
            String score1 = sc.replaceAll("&#xD;", "").replaceAll("\\\\n", "").trim();
            messageMap.put(item1, score1);
        }
        messageMap.remove("项目");
        String koubei = JSONObject.toJSON(messageMap).toString();
        String articleId = Pattern.compile("[^0-9]").matcher(koubeidetailsUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
        CrawlerData crawlerData = CrawlerData.builder()
                .data(crawlerRecord, page)
                .dataId(StringUtils.joinWith("-", domain(), article, articleId))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                .url(koubeidetailsUrl)
                .addContentKV(AICCommonField.Field_Title, title)
                .addContentKV(AICCommonField.Field_Content, koubei)
                .flowInPipelineTag("kafka")
                .build();
        crawlerData.setFilterPipelineResult(true);
        crawlerArticleDataList.add(crawlerData);
        return crawlerArticleDataList;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 10) {
                log.error("pcauto download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }
}
